{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "#data_df=pd.read_csv(\"train.csv\",nrows=100000000)\n",
    "data_df=pd.read_csv(\"train.csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "\n",
    "from datetime import datetime\n",
    "def feature1(data, test=False):\n",
    "    if test == 1:\n",
    "        dealed_data = data.drop_duplicates(['file_id'])['file_id']\n",
    "    else:\n",
    "        dealed_data = data.drop_duplicates(['file_id'])[['file_id','label']]\n",
    "    # 提取各类别特征的类别数\n",
    "    dealed_train = data.groupby(\"file_id\")[[\"api\",'tid','return_value']].nunique()\n",
    "    dealed_train.columns = ['api_nunique','tid_nunique','value_nunique']\n",
    "    \n",
    "    temp = data.groupby(['file_id'])['index'].count().rename('id_count')\n",
    "    dealed_train = pd.concat([dealed_train, temp],axis=1)\n",
    "\n",
    "    # 对每个file_id,的每个feat计数，然后求其统计特征\n",
    "    cate_feat = ['api','tid','return_value']\n",
    "    for feat in cate_feat:\n",
    "        temp = data.groupby(['file_id',feat])[feat].count().groupby(['file_id']).agg(['min','max','mean','median','std',pd.Series.mad,\n",
    "                                                                            pd.Series.skew,pd.Series.kurt]).add_prefix(feat+'_cnt_')\n",
    "        dealed_train = pd.concat([dealed_train, temp],axis=1)\n",
    "\n",
    "    # 提取交叉特征，每个file_id，每个不同api的tid种类数的统计特征\n",
    "    temp = data.groupby(['file_id','api'])['tid'].nunique().groupby('file_id').agg(['min','max','mean','median','std',pd.Series.mad,\n",
    "                                                                    pd.Series.skew,pd.Series.kurt]).add_prefix('api_tid_')\n",
    "    dealed_train = pd.concat([dealed_train, temp],axis=1)\n",
    "\n",
    "    # 提取交叉特征，每个file_id，每个不同api的return种类数的统计特征\n",
    "    temp = data.groupby(['file_id','api'])['return_value'].nunique().groupby('file_id').agg(['min','max','mean','median','std',pd.Series.mad,\n",
    "                                                                    pd.Series.skew,pd.Series.kurt]).add_prefix('api_value_')                                                               \n",
    "    dealed_train = pd.concat([dealed_train, temp],axis=1)\n",
    "\n",
    "    temp = data.groupby(['file_id','tid'])['api'].nunique().groupby('file_id').agg(['min','max','mean','median','std',pd.Series.mad,\n",
    "                                                                    pd.Series.skew,pd.Series.kurt]).add_prefix('tid_tid_')\n",
    "    dealed_train = pd.concat([dealed_train, temp],axis=1)\n",
    "\n",
    "    temp = data.groupby(['file_id','tid'])['return_value'].nunique().groupby('file_id').agg(['min','max','mean','median','std',pd.Series.mad,\n",
    "                                                                    pd.Series.skew,pd.Series.kurt]).add_prefix('tid_value_')                                                               \n",
    "    dealed_train = pd.concat([dealed_train, temp],axis=1)\n",
    "    return dealed_train\n",
    "\n",
    "train_data = feature1(data_df, test=False)\n",
    "#test_data = feature1(test, test=True)\n",
    "train_data.to_csv('train_featur_1mil.csv', index=False)\n",
    "#test_data.to_csv('test_feature1.csv', index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "label=data_df.groupby(['file_id','label'])['label'].unique()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "label.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "train_X=np.array(train_data)\n",
    "train_y=label\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_X.shape,train_y.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def runXGB(train_X,train_y,test_X,test_y=None,feature_names=None,seed_val=0,num_rounds=1000):\n",
    "    #参数设定\n",
    "    param = {}\n",
    "    param['objective'] = 'multi:softprob'#多分类、输出概率值\n",
    "    param['eta'] = 0.1#学习率\n",
    "    param['max_depth'] = 6#最大深度，越大越容易过拟合\n",
    "    param['silent'] = 1#打印提示信息\n",
    "    param['num_class'] = 6#三个类别\n",
    "    param['eval_metric']= \"mlogloss\"#对数损失\n",
    "    param['min_child_weight']=1#停止条件，这个参数非常影响结果，控制叶子节点中二阶导的和的最小值，该参数值越小，越容易 overfitting。\n",
    "    param['subsample'] =0.7#随机采样训练样本\n",
    "    param['colsample_bytree'] = 0.7# 生成树时进行的列采样\n",
    "    param['seed'] = seed_val#随机数种子\n",
    "    num_rounds = num_rounds#迭代次数\n",
    "    \n",
    "    plst = list(param.items())\n",
    "    xgtrain = xgb.DMatrix(train_X,label=train_y)\n",
    "    \n",
    "    if test_y is not None:\n",
    "        xgtest = xgb.DMatrix(test_X,label=test_y)\n",
    "        watchlist = [(xgtrain,'train'),(xgtest,'test')]\n",
    "        model = xgb.train(plst,xgtrain,num_rounds,watchlist,early_stopping_rounds=20)\n",
    "      #  early_stopping_rounds 当设置的迭代次数较大时，early_stopping_rounds 可在一定的迭代次数内准确率没有提升就停止训练\n",
    "    else:\n",
    "        xgtest = xgb.DMatrix(test_X)\n",
    "        model = xgb.train(plst,xgtrain,num_rounds)\n",
    "    pred_test_y = model.predict(xgtest)\n",
    "    return pred_test_y,model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import xgboost as xgb\n",
    "from sklearn import model_selection,preprocessing,ensemble\n",
    "from sklearn.cross_validation import train_test_split\n",
    "from sklearn.metrics import log_loss\n",
    "cv_scores = []\n",
    "x_train, x_valid, y_train, y_valid = train_test_split(train_X, train_y, test_size=0.2, random_state=0)\n",
    "pred_test_y,model=runXGB(x_train,y_train,x_valid,y_valid)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
